package com.bing.task;

import com.bing.utils.Help;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.UnsupportedMimeTypeException;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.SocketTimeoutException;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class Fetch implements Callable<Set<String>> {
    private static final Logger LOGGER = LoggerFactory.getLogger(Fetch.class);
    private static Pattern P_HTTP = Pattern.compile("(http|ftp|https):\\/\\/[\\w\\-_]+(\\.[\\w\\-_]+)+([\\w\\-\\.,@?^=%&amp;:/~\\+#]*[\\w\\-\\@?^=%&amp;/~\\+#])?");
    private String url;
    private String baseUrl;

    public Fetch(String url) {
        this.url = url;
        this.baseUrl = Help.getBaseUrl(url);
    }

    @Override
    public Set<String> call() throws Exception {
        Set<String> urls = new HashSet<>();
        try {
//            LOGGER.info("抓取网页:{}.baseUrl:{}", url, baseUrl);
            Document document = Jsoup.connect(url).get();
            for (Element a : document.select("a")) {
                String href = a.attr("href");
                if (!href.startsWith("javascript:") && !href.startsWith("#") && !StringUtil.isBlank(href)) {
                    if (!href.startsWith("http") && !href.startsWith("/")) {
                        href = baseUrl + href;
                    }
                    if (href.startsWith("/")) {
                        href = "http:/" + href;
                    }
                    urls.add(href);
                }
            }
            Matcher matcher = P_HTTP.matcher(document.html());
            while (matcher.find()) {
                final String group = matcher.group();
           /*     LOGGER.debug("找到链接: {}", group);*/
                urls.add(group);
            }
        } catch (UnsupportedMimeTypeException e) {
            //  LOGGER.info("非HTML");
        } catch (SocketTimeoutException e) {

        } catch (HttpStatusException e) {
        } catch (Exception e) {
            //   LOGGER.error("error: ", e);
        }
//        LOGGER.info("抓取网页:{}, 链接:{}", url, urls);
        return urls;
    }
}
